#!/usr/bin/env python
# coding=utf-8

import os
import sys
import codecs
import string
import logging
import glob

# set log
logFileName = 'FilterRoads.log'
def init_log():
    logger = logging.getLogger()
    hdlr = logging.FileHandler(logFileName)
    formatter = logging.Formatter('%(asctime)s %(levelname)s:: %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.INFO)
    return logger

log = init_log()

stdRoads = set(tuple(line.split()[0] for line in codecs.open(r'D:\ylb_work\VR\100kRoads\Places.txt', 'r', 'utf-8').readlines()))

def std_filter(roads):
	rds = []
	other = []
	for r in roads:
		if r in stdRoads:
			rds.append(r)
		else:
			other.append(r)
	return rds, other

def isroad(s):
	for c in s:
		if ord(c) < 256:
			return False
	if s.endswith(u'路') \
			or s.endswith(u'巷') \
			or s.endswith(u'大道') \
			or s.endswith(u'街道') \
			or s.endswith(u'国道') \
			or s.endswith(u'省道'):
		if s.count(u'路') > 1:
			return False
		else:
			return True
	else:
		return False


def filter_roads(roads):
	ret = []
	for r in roads:
		while r[-1].isspace():
			r = r[:-1]
		if isroad(r):
			ret.append(r)
	return ret

def remove_repeated(roads):
	roads.sort()
	repeatedPos = []
	for pos in range(2, len(roads)):
		if roads[pos] == roads[pos - 1]:
			repeatedPos.append(pos)
	repeatedPos.reverse()
	print(len(repeatedPos))
	for pos in repeatedPos:
		roads.pop(pos)
	return roads

def write_file(roads, filename):
	f = codecs.open(filename, 'w', 'utf-8')
	for r in roads:
		print>>f, r
	return len(roads)

def main():
	if len(sys.argv) != 3:
		print('Usage:')
		print('FilterRoads.py infile outfile')
		sys.exit(0)
	infilename = sys.argv[1]
	outfilename = sys.argv[2]
	infile = codecs.open(infilename, 'r', 'utf-8')
	roads = filter_roads(infile.readlines())
	roads = remove_repeated(roads)
	infile.close()
	write_file(roads, outfilename)
	rds, other = std_filter(roads)
	other.sort(key = lambda s: len(s))
	write_file(rds, 'rds.txt')
	write_file(other, 'other.txt')

	return True

if __name__ == '__main__':
	main()
	